import datetime
import scrapy
from bs4 import BeautifulSoup, Comment
from tuiqiuspiderPro.items import TuiqiuspiderproItem
from tuiqiuspiderPro.utils.base import delattrs
from tuiqiuspiderPro.utils.saveImge import SaveImg
from tuiqiuspiderPro.utils.parsertime import compTime


class SquawkaSpider(scrapy.Spider):
    name = 'squawka'
    allowed_domains = ['squawka.com']
    start_urls = ['https://www.squawka.com/en/news/']

    def parse(self, response, **kwargs):
        urls = response.xpath('.//a[@class="news-item__title-link"]/@href').extract()
        for url in urls:
            yield scrapy.Request(url=url, callback=self._article)

    def _article(self, response):
        item = TuiqiuspiderproItem()
        ts = response.xpath('.//div[@class="article-page__content"]/p/text()').extract_first()
        publish_time = self.parser_ts(ts.strip())
        if compTime(str(publish_time)):
            item['title'] = response.xpath('.//h1[@class="article-page__title"]/text()').extract_first()
            item['author'] = response.xpath('.//p[@class="article-page__meta"]/a/text()').extract_first()
            item['publish_time'] = publish_time
            content = response.xpath('.//div[@class="cms-content"]').extract_first()
            item['content'] = delattrs(content)
            item['source'] = "squawka"
            item['source_href'] = response.url
            item['tag'] = "足球综合"
            yield item

    def save_img(self, content):
        soup = BeautifulSoup(content, 'lxml')
        comments = soup.find_all(text=lambda text: isinstance(text, Comment))
        [comment.extract() for comment in comments]
        images = soup.find_all("img")
        for image in images:
            image['src'] = SaveImg(image['src'])
        return soup

    def parser_ts(self, ts):
        # ts = "Published: 12:58, 6 May 2021"
        ts_list = ts.split(",")
        H_time = ts_list[0].split(" ")
        after_ts = H_time[1] + " " + ts_list[1]
        publish_time = datetime.datetime.strptime(after_ts, "%H:%M %d %B %Y")
        return datetime.datetime.strftime(publish_time, "%Y-%m-%d %H:%M:%S")
